#!/usr/bin/python
#coding: utf-8

import sys
sys.path.append("..")

import time
from Logging.ProxyLogging import log_Main
from Config.Config import TimeSleep

class DownloaderPageHtml(object):
    def __init__(self, url, user_agent):
        self.url = url
        self.headers = {"user-agent" : user_agent}

    def Downloader(self, max_num):
        u"可以在Downloader模块中继续增加字段：可以搜索requests。get可以增加哪些字段即可"
        import requests
        # log_Main(u"防止ip被封禁，设置休眠%ds" % TimeSleep, "info")
        # time.sleep(TimeSleep)
        try:
            html = requests.get(self.url, headers= self.headers).text
        except Exception as e:
            html = None
            if max_num > 0:
                # log_Main("下载出错，继续进行第%d次尝试" % max_num, "error")
                html = self.Downloader(max_num - 1)
        if html is None and max_num == 0: # 什么都没有爬取到，给的链接正常，所以要继续进行爬取
            max_num = 5
            self.Downloader(max_num - 1)
        return html

if __name__ == "__main__":
    url = "http://www.kuaidaili.com/proxylist/1"
    user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36"
    print DownloaderPageHtml(url , user_agent).Downloader(5)